package crp.pipe;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;

import cc.mallet.pipe.Pipe;
import cc.mallet.types.FeatureSequenceWithBigrams;
import cc.mallet.types.Instance;
import cc.mallet.types.Token;
import cc.mallet.types.TokenSequence;

public class TokenSequenceRemoveNonTopicWords extends Pipe implements Serializable{
	HashSet<String> topicWords;
	/**
	 * 
	 */
	private static final long serialVersionUID = 1L;
	private static final int CURRENT_SERIAL_VERSION = 0;
	
	public TokenSequenceRemoveNonTopicWords(HashSet<String> topicWords){
		this.topicWords = topicWords;
	}
	
	public TokenSequenceRemoveNonTopicWords(File[] topicWordsFiles){
		this.topicWords = new HashSet<String>();
		for(File file : topicWordsFiles){
			String[] temp = fileToStringArray (file, null);
			for(String word : temp)
				this.topicWords.add(word);
		}
	}
	
	public Instance pipe (Instance carrier){
		TokenSequence ts = (TokenSequence) carrier.getData();
		TokenSequence ret = new TokenSequence ();
		for (int i = 0; i < ts.size(); i++) {
			Token t = ts.get(i);
			if(this.topicWords.contains(t.getText().toLowerCase()))
				ret.add(t);
		}
		carrier.setData(ret);
		return carrier;
	}
	
	private void writeObject (ObjectOutputStream out) throws IOException {
		out.writeInt (CURRENT_SERIAL_VERSION);
	}
	
	private void readObject (ObjectInputStream in) throws IOException, ClassNotFoundException {
		int version = in.readInt ();
	}
	
	private String[] fileToStringArray (File f, String encoding)
	{
		ArrayList<String> wordarray = new ArrayList<String>();

		try {

			BufferedReader input = null;
			if (encoding == null) {
				input = new BufferedReader (new FileReader (f));
			}
			else {
				input = new BufferedReader( new InputStreamReader( new FileInputStream(f), encoding ));
			}
			String line;

			while (( line = input.readLine()) != null) {
				String[] words = line.split ("\\s+");
				for (int i = 0; i < words.length; i++)
					wordarray.add (words[i]);
			}

		} catch (IOException e) {
			throw new IllegalArgumentException("Trouble reading file "+f);
		}
		return (String[]) wordarray.toArray(new String[]{});
	}

}
